package alp.starcode.sanping.crawler.impl;

import alp.starcode.sanping.common.BizConstant;
import alp.starcode.sanping.framework.database.mariadb.mybatis.entity.Information;
import org.apache.commons.lang3.time.DateUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;

/**
 * TODO 中国工信部资讯
 *
 * @author wujie
 * @date 2024/3/13 16:10
 */
public class HeBeiGxbServiceImpl implements PageProcessor {

    // 主页面URL集合
    static List<String> urlList = new ArrayList<>();
    static {
        urlList.add("https://gxt.hebei.gov.cn/hbgyhxxht/ztzl11/xfpgyspcxfz/spqgxxlhd/index.html");
        urlList.add("https://gxt.hebei.gov.cn/hbgyhxxht/ztzl11/xfpgyspcxfz/gjxfpgyspgz/index.html");
        urlList.add("https://gxt.hebei.gov.cn/hbgyhxxht/ztzl11/xfpgyspcxfz/hbsxfpgyspgz/index.html");
        urlList.add("https://gxt.hebei.gov.cn/hbgyhxxht/ztzl11/xfpgyspcxfz/spzlxgzc/index.html");
        urlList.add("https://gxt.hebei.gov.cn/hbgyhxxht/ztzl11/xfpgyspcxfz/szspcxfzds/index.html");
    }

    // 文章域名
    static String domain = "https://gxt.hebei.gov.cn";
    // 资讯页面URL集合
    static List<String> pageUrlList = new ArrayList<>();
    // 请求计数
    static int count = 0;

    private Site site;

    public HeBeiGxbServiceImpl() {
        // 配置爬虫的User-Agent和其他参数
        site = Site.me()
                .setUserAgent(BizConstant.SP_USER_AGENT)
                .setRetryTimes(BizConstant.SP_RETRY_TIMES)
                .setSleepTime(BizConstant.SP_SLEEP_TIME);
    }

    @Override
    public void process(Page page) {
        // 解析网页，提取需要的数据
        addTargetRequest(page);
        // 如果需要爬取其他页面，可以添加新的链接到Scheduler中
        count ++;
        if(count < pageUrlList.size()) {
            page.addTargetRequest(pageUrlList.get(count));
        }
    }

    public void addTargetRequest(Page page) {
        // 解析网页，提取需要的数据
        String title = page.getHtml().xpath("//h1/text()").get();
        String publishTime = page.getHtml().xpath("//h2/span[1]/text()").get().replace("发布时间：", "");
        String dataSource = page.getHtml().xpath("//h2/span[2]/text()").get().replace("    来源：", "");
        String content = page.getHtml().xpath("//div[@class='news_nr']/html()").get();
        content = content.replaceAll("src=\"/hbgyhxxht", "src=\""+domain+"/hbgyhxxht");

        Information information = new Information();
        information.setId(UUID.randomUUID().toString());
        information.setContent(content);
        information.setCreateUserName(BizConstant.SP_CREATE_BY);
        information.setDataSource(dataSource);
        information.setInfoType("通告");
        information.setIndustryType("消费品");
        try {
            publishTime = publishTime.replace("年", "-").replace("月", "-").replace("日", "");
            Date date = DateUtils.parseDate(publishTime, "yyyy-MM-dd");
            information.setPublishTime(date.getTime());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        information.setRegion("河北");
        information.setTitle(title);
        BizConstant.SP_INFORMATION_LIST.add(information);
    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 运行入口
     */
    public static void run() {
        // 清空缓存
        BizConstant.SP_INFORMATION_LIST = new ArrayList<>();

        for(String url : urlList) {
            try {
                Document doc = Jsoup.connect(url).get();

                Elements links = doc.select("li").select("a");
                for (Element link : links) {
                    String linkHref = link.attr("href");
                    if(linkHref.startsWith("/hbgyhxxht")) {
                        linkHref = domain + linkHref;
                    }
                    pageUrlList.add(linkHref);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        if(pageUrlList.size() > 0) {
            // 创建爬虫
            Spider spider = Spider.create(new HeBeiGxbServiceImpl())
                    .addUrl(pageUrlList.get(count))
                    .thread(5)
                    .setScheduler(new QueueScheduler());

            // 运行爬虫
            spider.run();
        }
    }

    public static void main(String[] args) {
        HeBeiGxbServiceImpl.run();
    }

}
